require File.expand_path('../../config/environment',  __FILE__)

require 'thread'

#Rails.logger = Logger.new $stdout

f = File.new(File.expand_path('../../data/iwanttobuy.txt',  __FILE__))
doc = Nokogiri::HTML(f, nil, 'UTF-8')

alink_tasks = Queue.new
doc.css('div.list h3 a').each do |alink|
  alink_tasks << alink['href']
end

threads = []
agent_aliases = Mechanize::AGENT_ALIASES.keys
agent_aliases.delete 'Mechanize'
3.times.each do
  threads << Thread.new do
    if Thread.current[:agent].nil?
      agent = Mechanize.new
      agent.user_agent_alias = agent_aliases[rand agent_aliases.size]
      Thread.current[:agent] = agent
    end
    while true
      begin
        break if alink_tasks.empty?
        cat_url = alink_tasks.pop
        ListTask.create_from_first cat_url
      rescue ActiveRecord::RecordInvalid => e
        Rails.logger.info "列表分类ID已经存在, #{cat_url}"
      rescue ThreadError => e
        Rails.logger.info "线程任务完成 #{Thread.current.object_id} , #{e.message}"
        break
      rescue Exception => e
        Rails.logger.warn "采集一级分类出错, #{cat_url}, #{e.message}, #{e.backtrace.join '\n'}"
      end
    end
  end
end

flag = true
while true
  threads.each do |th|
    if th.alive?
      flag = false 
      break
    end
  end
  exit if flag
  flag = true
  sleep 60
end
